{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ab8d51d6-7670-4cfb-87d1-f1c30fa2666e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from statistics import mean\n",
    "\n",
    "data = pd.read_csv(\"../../data/clean_weather.csv\", index_col=0)\n",
    "data = data.ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f3791a81-d488-4c7c-909b-8105c6b60587",
   "metadata": {},
   "outputs": [],
   "source": [
    "PREDICTORS = [\"tmax\", \"tmin\", \"rain\"]\n",
    "TARGET = \"tmax_tomorrow\"\n",
    "\n",
    "scaler = StandardScaler()\n",
    "data[PREDICTORS] = scaler.fit_transform(data[PREDICTORS])\n",
    "\n",
    "split_data = np.split(data, [int(.7*len(data)), int(.85*len(data))])\n",
    "(train_x, train_y), (valid_x, valid_y), (test_x, test_y) = [[d[PREDICTORS].to_numpy(), d[[TARGET]].to_numpy()] for d in split_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fc393270-36ea-4516-9bc7-a7ce51c59d76",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mse(actual, predicted):\n",
    "    return np.mean((actual-predicted)**2)\n",
    "\n",
    "def mse_grad(actual, predicted):\n",
    "    return (predicted - actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b1547940-59a1-40b6-8b7f-1ca15e414ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_layers(inputs):\n",
    "    layers = []\n",
    "    for i in range(1, len(inputs)):\n",
    "        layers.append([\n",
    "            np.random.rand(inputs[i-1], inputs[i]) / 5 - .1,\n",
    "            np.ones((1,inputs[i]))\n",
    "        ])\n",
    "    return layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f2a780eb-d517-4489-b19b-f6e319c11ded",
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(batch, layers):\n",
    "    hidden = [batch.copy()]\n",
    "    for i in range(len(layers)):\n",
    "        batch = np.matmul(batch, layers[i][0]) + layers[i][1]\n",
    "        if i < len(layers) - 1:\n",
    "            batch = np.maximum(batch, 0)\n",
    "        hidden.append(batch.copy())\n",
    "\n",
    "    return batch, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Hidden\n",
    "\n",
    "1,3\n",
    "1,10\n",
    "1,10\n",
    "\n",
    "## Output\n",
    "1,1\n",
    "\n",
    "## Weights\n",
    "\n",
    "3,10\n",
    "10,10\n",
    "10,1\n",
    "\n",
    "## Backwards\n",
    "\n",
    "### Iter 1 - layer 3\n",
    "\n",
    "* grad 1,1\n",
    "* w_grad 1,1 * 1,10 = 1,10 .T = 10,1\n",
    "* grad 10,1 * 1,1 = 10,1 .T = 1,10\n",
    "\n",
    "### Iter 2 - layer 2\n",
    "\n",
    "* grad 10,1\n",
    "* w_grad 10,1 * 1,10 = 10,10\n",
    "* grad 10,10 10,1 = 10,1 . T = 1,10\n",
    "\n",
    "## Iter 3 - layer 1\n",
    "\n",
    "* grad 10,1\n",
    "* w_grad 10,1 1,3 = 10,3 .T = 3,10\n",
    "* grad 3,10 10,1 = 3,1 .T = 1,3"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9b48ba67-863e-49ab-97f1-85322a28c913",
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward(layers, hidden, grad, lr):\n",
    "    for i in range(len(layers)-1, -1, -1):\n",
    "        if i != len(layers) - 1:\n",
    "            grad = np.multiply(grad, np.heaviside(hidden[i+1], 0))\n",
    "\n",
    "        grad = grad.T\n",
    "        w_grad = np.matmul(grad, hidden[i]).T\n",
    "        b_grad = np.mean(grad.T, axis=0)\n",
    "\n",
    "        layers[i][0] -= (w_grad + layers[i][0] * .01) * lr\n",
    "        layers[i][1] -= b_grad * lr\n",
    "        grad = np.matmul(layers[i][0], grad).T\n",
    "    return layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c76220dc-15c3-4b46-83af-7d644c47d51c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Train MSE: 3298.8488090041587 Valid MSE: 1699.191865893834\n",
      "Epoch: 1 Train MSE: 347.13026842327326 Valid MSE: 25.506153413026723\n",
      "Epoch: 2 Train MSE: 23.084596669639275 Valid MSE: 21.19942451555908\n",
      "Epoch: 3 Train MSE: 22.388523703717826 Valid MSE: 20.833607483123387\n",
      "Epoch: 4 Train MSE: 22.184622048882105 Valid MSE: 20.762662004702822\n",
      "Epoch: 5 Train MSE: 22.1317309020479 Valid MSE: 20.75264472565585\n",
      "Epoch: 6 Train MSE: 22.116946070297494 Valid MSE: 20.75349128500077\n",
      "Epoch: 7 Train MSE: 22.111837612133225 Valid MSE: 20.75512212748549\n",
      "Epoch: 8 Train MSE: 22.109293717522267 Valid MSE: 20.75600995109669\n",
      "Epoch: 9 Train MSE: 22.107265980091956 Valid MSE: 20.755970386694667\n"
     ]
    }
   ],
   "source": [
    "layer_conf = [3,10,10,1]\n",
    "lr = 1e-6\n",
    "epochs=10\n",
    "batch_size = 8\n",
    "\n",
    "layers = init_layers(layer_conf)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    epoch_loss = []\n",
    "\n",
    "    for i in range(0, train_x.shape[0], batch_size):\n",
    "        x_batch = train_x[i:(i+batch_size)]\n",
    "        y_batch = train_y[i:(i+batch_size)]\n",
    "        pred, hidden = forward(x_batch, layers)\n",
    "\n",
    "        loss = mse_grad(y_batch, pred)\n",
    "        epoch_loss.append(np.mean(loss ** 2))\n",
    "\n",
    "        layers = backward(layers, hidden, loss, lr)\n",
    "    \n",
    "\n",
    "    valid_preds, _ = forward(valid_x, layers)\n",
    "\n",
    "    print(f\"Epoch: {epoch} Train MSE: {mean(epoch_loss)} Valid MSE: {mse(valid_preds,valid_y)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
